{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import operator\n",
    "main_path = '/content/drive/My Drive/Colab Notebooks/'\n",
    "eng = np.load('pad_eng.npy')\n",
    "cns = np.load('pad_cns.npy')\n",
    "cns_o = np.load('cns_o.npy', allow_pickle=True)\n",
    "with open('vocab_bag.pkl', 'rb') as f:\n",
    "    words = pickle.load(f)\n",
    "with open('pad_word_to_index.pkl', 'rb') as f:\n",
    "    word_to_index = pickle.load(f)\n",
    "with open('pad_index_to_word.pkl', 'rb') as f:\n",
    "    index_to_word = pickle.load(f)\n",
    "vocab_size = len(word_to_index) + 1\n",
    "maxLen=20\n",
    "def get_file_list(file_path):\n",
    "    dir_list = os.listdir(file_path)\n",
    "    if not dir_list:\n",
    "        return\n",
    "    else:\n",
    "        dir_list = sorted(dir_list, key=lambda x: os.path.getmtime(os.path.join(file_path, x)))\n",
    "    return dir_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing import sequence\n",
    "def generate_train(batch_size):\n",
    "    print('\\n*********************************generate_train()*********************************')\n",
    "    steps=0\n",
    "    eng_ = eng\n",
    "    cns_ = cns\n",
    "    while True:\n",
    "        batch_cns_o = cns_o[steps:steps+batch_size]\n",
    "        batch_eng = eng_[steps:steps+batch_size]\n",
    "        batch_cns = cns_[steps:steps+batch_size]\n",
    "        outs = np.zeros([batch_size, maxLen, vocab_size], dtype='float32')\n",
    "        for pos, i in enumerate(batch_cns_o):\n",
    "            for pos_, j in enumerate(i):\n",
    "                if pos_ > 20:\n",
    "                    print(i)\n",
    "                outs[pos, pos_, j] = 1 # one-hot\n",
    "        yield [batch_eng, batch_cns], outs\n",
    "        steps += batch_size\n",
    "        if steps == 20000:\n",
    "            steps = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\users\\王嘉意\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_cns (InputLayer)          (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_eng (InputLayer)          (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 100)    1775200     input_eng[0][0]                  \n",
      "                                                                 input_cns[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "encoder_lstm (LSTM)             [(None, None, 256),  365568      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "decoder_lstm (LSTM)             [(None, None, 256),  365568      embedding_1[1][0]                \n",
      "                                                                 encoder_lstm[0][1]               \n",
      "                                                                 encoder_lstm[0][2]               \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, None, None)   0           decoder_lstm[0][0]               \n",
      "                                                                 encoder_lstm[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, None, None)   0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, None, 256)    0           activation_1[0][0]               \n",
      "                                                                 encoder_lstm[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, 512)    0           dot_2[0][0]                      \n",
      "                                                                 decoder_lstm[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 128)    65664       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, None, 17752)  2290008     time_distributed_1[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 4,862,008\n",
      "Trainable params: 4,862,008\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Embedding\n",
    "from keras.layers import Input, Dense, LSTM, TimeDistributed, Bidirectional, Dropout, Concatenate, RepeatVector, Activation, Dot\n",
    "from keras.layers import concatenate, dot                    \n",
    "from keras.models import Model\n",
    "from keras.utils import plot_model\n",
    "from keras.callbacks import ModelCheckpoint, TensorBoard,ReduceLROnPlateau\n",
    "from keras.initializers import TruncatedNormal\n",
    "import pydot\n",
    "import os, re\n",
    "truncatednormal = TruncatedNormal(mean=0.0, stddev=0.05)\n",
    "embed_layer = Embedding(input_dim=vocab_size, \n",
    "                        output_dim=100, \n",
    "                        mask_zero=True,\n",
    "                        input_length=None,\n",
    "                        embeddings_initializer= truncatednormal)\n",
    "LSTM_encoder = LSTM(256,\n",
    "                      return_sequences=True,\n",
    "                      return_state=True,\n",
    "#                       activation='relu',\n",
    "#                       dropout=0.25,\n",
    "#                       recurrent_dropout=0.1,\n",
    "                      kernel_initializer= 'lecun_uniform',\n",
    "                      name='encoder_lstm'\n",
    "                        )\n",
    "LSTM_decoder = LSTM(256, \n",
    "                    return_sequences=True, \n",
    "                    return_state=True, \n",
    "#                     activation = 'relu',\n",
    "#                     dropout=0.25, \n",
    "#                     recurrent_dropout=0.1,\n",
    "                    kernel_initializer= 'lecun_uniform',\n",
    "                    name='decoder_lstm'\n",
    "                   )\n",
    "#encoder输入 与 decoder输入\n",
    "input_eng = Input(shape=(None, ), dtype='int32', name='input_eng')\n",
    "input_cns = Input(shape=(None, ), dtype='int32', name='input_cns')\n",
    "\n",
    "input_eng_embed = embed_layer(input_eng)\n",
    "input_cns_embed = embed_layer(input_cns)\n",
    "\n",
    "encoder_lstm, eng_h, eng_c = LSTM_encoder(input_eng_embed)\n",
    "\n",
    "decoder_lstm, _, _ = LSTM_decoder(input_cns_embed, \n",
    "                                  initial_state=[eng_h, eng_c])\n",
    "\n",
    "attention = dot([decoder_lstm, encoder_lstm], axes=[2, 2])\n",
    "attention = Activation('softmax')(attention)\n",
    "context = dot([attention, encoder_lstm], axes=[2,1])\n",
    "decoder_combined_context = concatenate([context, decoder_lstm])\n",
    "\n",
    "# output = dense1(decoder_combined_context)\n",
    "# output = dense2(Dropout(0.5)(output))\n",
    "\n",
    "# Has another weight + tanh layer as described in equation (5) of the paper\n",
    "decoder_dense1 = TimeDistributed(Dense(128,activation=\"tanh\"))\n",
    "decoder_dense2 = TimeDistributed(Dense(vocab_size,activation=\"softmax\"))\n",
    "output = decoder_dense1(decoder_combined_context) # equation (5) of the paper\n",
    "output = decoder_dense2(output) # equation (6) of the paper\n",
    "\n",
    "model = Model([input_eng, input_cns], output)\n",
    "\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n",
    "\n",
    "model.load_weights('models/W--189-0.5900-.h5')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_eng (InputLayer)       (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, None, 100)         1775200   \n",
      "_________________________________________________________________\n",
      "encoder_lstm (LSTM)          [(None, None, 256), (None 365568    \n",
      "=================================================================\n",
      "Total params: 2,140,768\n",
      "Trainable params: 2,140,768\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_cns (InputLayer)          (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 100)    1775200     input_cns[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 256)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 256)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "decoder_lstm (LSTM)             [(None, None, 256),  365568      embedding_1[1][0]                \n",
      "                                                                 input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            (None, 20, 256)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dot_3 (Dot)                     (None, None, 20)     0           decoder_lstm[1][0]               \n",
      "                                                                 input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, None, 20)     0           dot_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_4 (Dot)                     (None, None, 256)    0           activation_2[0][0]               \n",
      "                                                                 input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, None, 512)    0           dot_4[0][0]                      \n",
      "                                                                 decoder_lstm[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 128)    65664       concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, None, 17752)  2290008     time_distributed_1[1][0]         \n",
      "==================================================================================================\n",
      "Total params: 4,496,440\n",
      "Trainable params: 4,496,440\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "eng_model = Model(input_eng, [encoder_lstm, eng_h, eng_c])\n",
    "eng_model.summary()\n",
    "cns_h = Input(shape=(256,))\n",
    "cns_c = Input(shape=(256,))\n",
    "encoder_lstm = Input(shape=(maxLen,256))\n",
    "target, h, c = LSTM_decoder(input_cns_embed, initial_state=[cns_h, cns_c])\n",
    "attention = dot([target, encoder_lstm], axes=[2, 2])\n",
    "attention_ = Activation('softmax')(attention)\n",
    "context = dot([attention_, encoder_lstm], axes=[2,1])\n",
    "decoder_combined_context = concatenate([context, target])\n",
    "output = decoder_dense1(decoder_combined_context) # equation (5) of the paper\n",
    "output = decoder_dense2(output) # equation (6) of the paper\n",
    "cns_model = Model([input_cns, cns_h, cns_c, encoder_lstm], [output, h, c, attention_])\n",
    "cns_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import jieba\n",
    "import requests\n",
    "def input_eng(seq):\n",
    "    seq = seq.split(' ')\n",
    "    sentence = seq\n",
    "    seq = np.array([word_to_index[w] for w in seq])\n",
    "    seq = sequence.pad_sequences([seq], maxlen=maxLen,\n",
    "                                          padding='post', truncating='post')\n",
    "    print(seq)\n",
    "    return seq, sentence\n",
    "def decode_greedy(seq, sentence):\n",
    "    eng = seq\n",
    "    cns = np.zeros((1, 1))\n",
    "    attention_plot = np.zeros((20, 20))\n",
    "    cns[0, 0] = word_to_index['BOS']\n",
    "    i=1\n",
    "    cns_ = []\n",
    "    flag = 0\n",
    "    encoder_lstm_, eng_h, eng_c = eng_model.predict(x=eng, verbose=1)\n",
    "#     print(eng_h, '\\n')\n",
    "    while flag != 1:\n",
    "        prediction, prediction_h, prediction_c, attention = cns_model.predict([\n",
    "            cns, eng_h, eng_c, encoder_lstm_\n",
    "        ])\n",
    "        attention_weights = attention.reshape(-1, )\n",
    "        attention_plot[i] = attention_weights\n",
    "        word_arg = np.argmax(prediction[0, -1, :])#\n",
    "        cns_.append(index_to_word[word_arg])\n",
    "        if word_arg == word_to_index['EOS']  or i > 20:\n",
    "            flag = 1\n",
    "        cns = np.zeros((1, 1))\n",
    "        cns[0, 0] = word_arg\n",
    "        eng_h = prediction_h\n",
    "        eng_c = prediction_c\n",
    "        i += 1\n",
    "    result = ' '.join(cns_)\n",
    "#     attention_plot = attention_plot[:len(result.split(' ')), :len(sentence)]\n",
    "#     plot_attention(attention_plot, sentence, result.split(' '))\n",
    "    return ' '.join(cns_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Where are you ?\n",
      "[[257 258  62  25   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0]]\n",
      "['Where', 'are', 'you', '?']\n",
      "1/1 [==============================] - 0s 135ms/step\n",
      "翻译:  你 在 哪儿 ? EOS\n",
      "I love you !\n",
      "[[  7 219  62   5   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0]]\n",
      "['I', 'love', 'you', '!']\n",
      "1/1 [==============================] - 0s 4ms/step\n",
      "翻译:  我 爱 你 。 EOS\n",
      "Do you love me ?\n",
      "[[114  62 219  30  25   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0]]\n",
      "['Do', 'you', 'love', 'me', '?']\n",
      "1/1 [==============================] - 0s 3ms/step\n",
      "翻译:  你 爱 我 吗 ？ EOS\n",
      "x\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    seq = input()\n",
    "    if seq == 'x':\n",
    "        break\n",
    "    seq, sentence = input_eng(seq)\n",
    "    print(sentence)\n",
    "    cns = decode_greedy(seq, sentence)\n",
    "    print('翻译: ', cns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
